{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.)\n",
      "tensor(1.)\n",
      "tensor(1.)\n",
      "w:  Parameter containing:\n",
      "tensor([[-0.0430, -0.1551,  0.4134],\n",
      "        [-0.5059,  0.2654,  0.0162]], requires_grad=True)\n",
      "b:  Parameter containing:\n",
      "tensor([-0.0426, -0.3433], requires_grad=True)\n",
      "loss:  1.009313702583313\n",
      "dL/dw:  tensor([[ 0.4397,  0.1407,  0.2419],\n",
      "        [-0.4108,  0.2044,  0.6520]])\n",
      "dL/db:  tensor([-0.1729, -0.3978])\n",
      "loss after 1 step optimization:  0.9984379410743713\n",
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../../data/cifar-10-python.tar.gz\n",
      "torch.Size([3, 32, 32])\n",
      "6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to C:\\Users\\haigu/.torch\\models\\resnet18-5c106cde.pth\n",
      "100%|███████████████████████████████████████████████████████████████████| 46827520/46827520 [25:55<00:00, 30114.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 100])\n"
     ]
    }
   ],
   "source": [
    "# %load main.py\n",
    "import torch \n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                         Table of Contents                          #\n",
    "# ================================================================== #\n",
    "\n",
    "# 1. Basic autograd example 1               (Line 25 to 39)\n",
    "# 2. Basic autograd example 2               (Line 46 to 83)\n",
    "# 3. Loading data from numpy                (Line 90 to 97)\n",
    "# 4. Input pipline                          (Line 104 to 129)\n",
    "# 5. Input pipline for custom dataset       (Line 136 to 156)\n",
    "# 6. Pretrained model                       (Line 163 to 176)\n",
    "# 7. Save and load model                    (Line 183 to 189) \n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                     1. Basic autograd example 1                    #\n",
    "# ================================================================== #\n",
    "\n",
    "# Create tensors.\n",
    "x = torch.tensor(1., requires_grad=True)\n",
    "w = torch.tensor(2., requires_grad=True)\n",
    "b = torch.tensor(3., requires_grad=True)\n",
    "\n",
    "# Build a computational graph.\n",
    "y = w * x + b    # y = 2 * x + 3\n",
    "\n",
    "# Compute gradients.\n",
    "y.backward()\n",
    "\n",
    "# Print out the gradients.\n",
    "print(x.grad)    # x.grad = 2 \n",
    "print(w.grad)    # w.grad = 1 \n",
    "print(b.grad)    # b.grad = 1 \n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                    2. Basic autograd example 2                     #\n",
    "# ================================================================== #\n",
    "\n",
    "# Create tensors of shape (10, 3) and (10, 2).\n",
    "x = torch.randn(10, 3)\n",
    "y = torch.randn(10, 2)\n",
    "\n",
    "# Build a fully connected layer.\n",
    "linear = nn.Linear(3, 2)\n",
    "print ('w: ', linear.weight)\n",
    "print ('b: ', linear.bias)\n",
    "\n",
    "# Build loss function and optimizer.\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.SGD(linear.parameters(), lr=0.01)\n",
    "\n",
    "# Forward pass.\n",
    "pred = linear(x)\n",
    "\n",
    "# Compute loss.\n",
    "loss = criterion(pred, y)\n",
    "print('loss: ', loss.item())\n",
    "\n",
    "# Backward pass.\n",
    "loss.backward()\n",
    "\n",
    "# Print out the gradients.\n",
    "print ('dL/dw: ', linear.weight.grad) \n",
    "print ('dL/db: ', linear.bias.grad)\n",
    "\n",
    "# 1-step gradient descent.\n",
    "optimizer.step()\n",
    "\n",
    "# You can also perform gradient descent at the low level.\n",
    "# linear.weight.data.sub_(0.01 * linear.weight.grad.data)\n",
    "# linear.bias.data.sub_(0.01 * linear.bias.grad.data)\n",
    "\n",
    "# Print out the loss after 1-step gradient descent.\n",
    "pred = linear(x)\n",
    "loss = criterion(pred, y)\n",
    "print('loss after 1 step optimization: ', loss.item())\n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                     3. Loading data from numpy                     #\n",
    "# ================================================================== #\n",
    "\n",
    "# Create a numpy array.\n",
    "x = np.array([[1, 2], [3, 4]])\n",
    "\n",
    "# Convert the numpy array to a torch tensor.\n",
    "y = torch.from_numpy(x)\n",
    "\n",
    "# Convert the torch tensor to a numpy array.\n",
    "z = y.numpy()\n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                         4. Input pipline                           #\n",
    "# ================================================================== #\n",
    "\n",
    "# Download and construct CIFAR-10 dataset.\n",
    "train_dataset = torchvision.datasets.CIFAR10(root='../../data/',\n",
    "                                             train=True, \n",
    "                                             transform=transforms.ToTensor(),\n",
    "                                             download=True)\n",
    "\n",
    "# Fetch one data pair (read data from disk).\n",
    "image, label = train_dataset[0]\n",
    "print (image.size())\n",
    "print (label)\n",
    "\n",
    "# Data loader (this provides queues and threads in a very simple way).\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=64, \n",
    "                                           shuffle=True)\n",
    "\n",
    "# When iteration starts, queue and thread start to load data from files.\n",
    "data_iter = iter(train_loader)\n",
    "\n",
    "# Mini-batch images and labels.\n",
    "images, labels = data_iter.next()\n",
    "\n",
    "# Actual usage of the data loader is as below.\n",
    "for images, labels in train_loader:\n",
    "    # Training code should be written here.\n",
    "    pass\n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                5. Input pipline for custom dataset                 #\n",
    "# ================================================================== #\n",
    "\n",
    "# You should build your custom dataset as below.\n",
    "class CustomDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self):\n",
    "        # TODO\n",
    "        # 1. Initialize file paths or a list of file names. \n",
    "        pass\n",
    "    def __getitem__(self, index):\n",
    "        # TODO\n",
    "        # 1. Read one data from file (e.g. using numpy.fromfile, PIL.Image.open).\n",
    "        # 2. Preprocess the data (e.g. torchvision.Transform).\n",
    "        # 3. Return a data pair (e.g. image and label).\n",
    "        pass\n",
    "    def __len__(self):\n",
    "        # You should change 0 to the total size of your dataset.\n",
    "        return 0 \n",
    "\n",
    "# You can then use the prebuilt data loader. \n",
    "custom_dataset = CustomDataset()\n",
    "train_loader = torch.utils.data.DataLoader(dataset=custom_dataset,\n",
    "                                           batch_size=64, \n",
    "                                           shuffle=True)\n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                        6. Pretrained model                         #\n",
    "# ================================================================== #\n",
    "\n",
    "# Download and load the pretrained ResNet-18.\n",
    "resnet = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "# If you want to finetune only the top layer of the model, set as below.\n",
    "for param in resnet.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "# Replace the top layer for finetuning.\n",
    "resnet.fc = nn.Linear(resnet.fc.in_features, 100)  # 100 is an example.\n",
    "\n",
    "# Forward pass.\n",
    "images = torch.randn(64, 3, 224, 224)\n",
    "outputs = resnet(images)\n",
    "print (outputs.size())     # (64, 100)\n",
    "\n",
    "\n",
    "# ================================================================== #\n",
    "#                      7. Save and load the model                    #\n",
    "# ================================================================== #\n",
    "\n",
    "# Save and load the entire model.\n",
    "torch.save(resnet, 'model.ckpt')\n",
    "model = torch.load('model.ckpt')\n",
    "\n",
    "# Save and load only the model parameters (recommended).\n",
    "torch.save(resnet.state_dict(), 'params.ckpt')\n",
    "resnet.load_state_dict(torch.load('params.ckpt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
